package top.foxhome.web.spiders;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import top.foxhome.web.spiders.document_process.IDocumentProcess;
import top.foxhome.web.spiders.util.UrlUtil;
import top.foxhome.web.spiders.web.WebEngine;

/**
 * 抓取任务
 */
public class SpidersTask {
    private String host;
    private BrowsCache mBrowsCache;
    private WebEngine webEngine;
    private IDocumentProcess mDocumentProcess;

    public SpidersTask(String host, IDocumentProcess documentProcess) {
        this.host = host;
        webEngine = new WebEngine();
        mBrowsCache = new BrowsCache();
        mDocumentProcess = documentProcess;
    }

    /**
     * 执行
     */
    public void execute() {
        mBrowsCache.clear();
        executeImp(host);
    }

    /**
     * 递归浏览页面实现
     *
     * @param url
     */
    private void executeImp(String url) {
        if (mBrowsCache.contains(url)) return;
        System.out.println(url);
        int reTry = 15;
        Document mElement = null;
        do {
            mElement = webEngine.browse(url);
            reTry--;
        } while (mElement == null && reTry > 0);
        if (mElement == null) return;
        mDocumentProcess.onDocument(host, url, mElement);
        //将已解析网址进行记录
        mBrowsCache.addUrl(url);
        Element bodyElement = mElement.body();
        Elements aElements = bodyElement.getElementsByTag("a");
        //迭代元素
        aElements.forEach(element -> {
            String href = element.attr("href");
            href = UrlUtil.buildUrl(href, host, host);
            if (href == null) return;
            if (!UrlUtil.isOffSiteLinks(href, host) && !mBrowsCache.contains(href)) {
                executeImp(href);
            }
        });

    }
}
